#encoding: utf-8
from pycw2vec.config.word2vec_config import BASE_DIR
import re
from pathlib import Path
import codecs
import sys
import csv
import os
import codecs

base_dir = Path("/home/stu/Documents/dataset/sougo/sougoCA_full")

def split_file(base_dir,input_file,max_lines):
    f = open(base_dir / input_file,'r')
    line_index = 0 #每个txt行数
    txt_num = 0  #txt数目
    output = open(base_dir / "split_files/{}.txt".format(txt_num), 'w+')
    for line in f:
        if "</content>" in line and line_index > max_lines: #一篇文章结束且达到此t x t最大行
            #写入此t x t最后一行，并开一篇新t x t，行数重设为0
            output.write(line + "\n")
            output.close()
            txt_num += 1
            output = open(base_dir / "split_files/{}.txt".format(txt_num), 'w+')
            line_index = 0

            print(txt_num)
        else:
            #写入此txt
            output.write(line + "\n")
            line_index += 1

def extractContent(rootdir,output_file):
    files = os.listdir(rootdir)
    pattern1 = r'<content>(.*?)</content>'
    output = open(output_file,'w+',encoding='utf8')
    for x in range(0,len(files)):
        path = rootdir / files[x] # 获取当前文件路径
        if os.path.isfile(path) and files[x].endswith(".txt"):
            print(path)
            content = open(path,'r').read()
            texts = re.findall(pattern1,content,re.S)
            for text in texts:
                # print(text)
                output.write(text + "\n")
            # sys.exit(0)
    output.close()

if __name__ =="__main__":
    # split_file(base_dir, "news.txt", 100000)
    extractContent(base_dir / 'split_files',base_dir / 'sougoCA_full.txt')